/*
 * Copyright (C) 2012 JiangHongTiao
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package sk.lieskove.jianghongtiao.paris.core.classify;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import sk.lieskove.jianghongtiao.common.utils.FileUtils;
import sk.lieskove.jianghongtiao.common.utils.PropertiesUtils;
import sk.lieskove.jianghongtiao.websearch.persistence.ClassificationStorage;
import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.filters.Filter;
import weka.filters.unsupervised.attribute.RemoveByName;
import weka.filters.unsupervised.attribute.StringToWordVector;

/**
 * 
 * @author xjuraj e-mail: jjurco.sk_gmail.com
 */
public class ThreadArffMaker extends Thread implements Serializable {

    private transient Logger log = Logger.getLogger(ThreadArffMaker.class);
    private transient PropertiesUtils pu = new PropertiesUtils(ThreadArffMaker.class);
    private Map<String, List<ClassificationStorage>> learnInstances = null;
    private Instances instances = null;
    private List<String> languages = null;
    private int wordsToKeep = 1000;

    public ThreadArffMaker(Map<String, List<ClassificationStorage>> learnInstances, String name,
            List<String> languages, int wordsToKeep) {
        super(name);
        this.learnInstances = learnInstances;
        this.languages = languages;
        this.wordsToKeep = wordsToKeep;
    }

    @Override
    public void run() {
        Instances createInstances = createInstances(languages, learnInstances);
        instances = modifyInstances(createInstances, wordsToKeep);
    }

    /**
     * create learn instances
     * @param languages list of languages in learning set
     * @param learnInstances learn instances
     * @return instances in DB friendly format
     */
    public Instances createInstances(List<String> languages,
            Map<String, List<ClassificationStorage>> learnInstances) {

        if ((learnInstances == null) || (learnInstances.isEmpty())) {
            return null;
        }
        try {
            Instance i = new DenseInstance(4);
            Attribute id = new Attribute("ID");
            id.setWeight(0);
            Attribute pageClass = new Attribute("page_class", new ArrayList<String>(learnInstances.keySet()));
            Attribute pageLang = new Attribute("page_language", languages);
            Attribute pageContent = new Attribute("page_content", (List<String>) null);

            ArrayList<Attribute> atts = new ArrayList<Attribute>(4);
            atts.add(id);
            atts.add(pageClass);
            atts.add(pageLang);
            atts.add(pageContent);

            // 3. fill with data
            Instances data = new Instances("Paris", atts, 0);
            for (String docClass : learnInstances.keySet()) {
                List<ClassificationStorage> list = learnInstances.get(docClass);
                for (ClassificationStorage cs : list) {
                    String textData = FileUtils.read(new File(cs.getWebSearchResponse().getFileLocation()));
                    if (textData == null) {
                        textData = "null";
                    }

                    String lang = (cs.getWebSearchResponse().getLanguageName() != null)
                            ? cs.getWebSearchResponse().getLanguageName().name() : "null";

                    i.setValue(id, cs.getId());
                    i.setValue(pageClass, docClass);
                    i.setValue(pageLang, languages.indexOf(lang));
                    i.setValue(pageContent, textData);
                    
                    i.setDataset(data);
                    
                    data.add(i);
                }
            }

            data.setClass(pageClass);
            return data;
        } catch (IOException ex) {
            log.error("Cannot save ARFF file.");
        }
        return null;
    }

    /**
     * apply filters on instances
     * @param data instances retrieved from database 
     * @param wordsToKeep words to keep with method StringToWordVector
     * @return modified learn instances - applied filters
     */
    public Instances modifyInstances(Instances data, int wordsToKeep) {
        StringToWordVector stringToWordVector = new StringToWordVector();
        File sw = new File("/home/xjuraj/Projects/master-thesis/Paris/resources"
                + "/stopwords/inf-man-stop-words.cs");
        RemoveByName removeByName = new RemoveByName();
        if (sw.exists()) {
            stringToWordVector.setStopwords(sw);
        } else {
            log.warn("Stop words not found! Filename: " + sw.getAbsolutePath());
        }
        stringToWordVector.setWordsToKeep(wordsToKeep);
        Instances result = null;
        try {
            stringToWordVector.setInputFormat(data);
            result = Filter.useFilter(data, stringToWordVector);

            removeByName.setExpression("(\\d+)|(.)");
            removeByName.setInvertSelection(false);
            removeByName.setInputFormat(result);
            result = Filter.useFilter(result, removeByName);
        } catch (Exception ex) {
            log.error("Exception thrown while modifying instances with StringToWordVector");
        }
        return result;
    }

    public Instances getInstances() {
        return instances;
    }
}
